import urllib
import traceback
from bs4 import BeautifulSoup
import urllib.parse
import re
import json
import warnings
import random
import unicodedata
from tqdm import tqdm
warnings.filterwarnings('ignore')

        
def get_map(data_path):
    data_map = dict()
    with open(data_path, encoding='utf-8') as f:
        for line in f:
            split = line.strip('\n').split('\t')
            if len(split) > 1:
                data_map[split[0]] = split[1]
            else:
                data_map[split[0]] = ''
    return data_map
    
    
re_replace_pair = [
    # 删除公式和字体修饰
    (r'\\overset\s*\{\s*\\lower0.5em\s*\\hbox\{\$\\smash\{\\scriptscriptstyle\\rightharpoonup\}\$\}\}',r'\\overrightarrow'),
    (r'\\mathord\{\\buildrel\{\\lower3pt\\hbox\{\$\\scriptscriptstyle\\frown\$\}\}\+\\over\+(.+?)\}',r'\1'),
    (r'\{\\raise0\.5ex\\hbox\{\$\\scriptstyle\+0\$\}\\kern-0\.1em\/\\kern-0\.15em\\lower0\.25ex\\hbox\{\$\\scriptstyle\+0\$\}\}',r'\%'),
    (r'\\sideset\s*\{(.*?)\}\{(.*?)\}\{sqrt\{x\}\}',r'\1sqrt\{x\}\2'),
    (r'\\sideset\s*\{(.*?)\}\{(.*?)\}\{(.*?)\}',r'\1\3\2'),
    (r'_\{\}\^\{0\}\\textrm\{C\}|\\circC', r'\\textcelsius'),
    (r'(\\overrightarrow|\\underline|\\bf|\\text|\\textrm|\\boldsymbol|\\operatorname|\\fbox)\s*\{(.*?)\}', r'\2'),
    (r'\\math(?:bb|bf|bfit|cal|frak|sf|sfbf|sfbfit|sfit|op|rm|tt)\s*\{(.*?)\}', r'\1'),
    (r'\\right\s*\.', r''),
    (r'(\\nolimits|\\limits)\s*', r''),
    (r'(\\left|\\right)\s*([\(\[\{\|])', r'\2'),
    (r'\\left\s*\\\s*\{', r''),
    (r'\\boxed|\\buildrel|\\displaystyle|\\dpi\s*\{100\}|\\rm|\\middle|\\textstyle|\\scriptstyle|\\scriptscriptstyle|\\small|\{\\kern\s*\+1pt\}|\{\*\{35\}\{l\}\}|\\mathchar\'26\\mkern-10mu', r''),
    (r'(\\hspace|\\mspace|\\phantom)\s*\{(.*?)\}', r''),
    # 修正表达
  	(r'\\ast',r'*'),
  	(r'\\cap',r'\\bigcap'),
  	(r'\\geqslant',r'\\ge'),
  	(r'\\left\(',r'('),
  	(r'\\left\[',r'['),
  	(r'\\left\|',r'|'),
  	(r'\\right\)',r')'),
  	(r'\\right\]',r']'),
  	(r'\\right\|',r'|'),
  	(r'\\leqslant',r'\\le'),
  	(r'\\neq',r'\\ne'),
  	(r'\\not=',r'\\ne'),
  	(r'\\overset\{¯\}',r'\\overline'),
  	(r'\\overset\{-\}',r'\\overline'),
  	(r'\\overset\{→\}',r'\\overrightarrow'),
  	(r'\\overset\{∧\}',r'\\hat'),
  	(r'\\overset\{\\hat\{\}\}',r'\\hat'),
  	(r'\\widehat',r'\\hat'),
  	(r'\\hat',r'\^'),
  	(r'\^\{\\circ\}C',r'\\textcelsius'),
  	(r'\^\{\\circ\}c',r'\\textcelsius'),
  	(r'\^\{\\prime\}',r'\''),
  	(r'\^\{\\\'\}',r'\\\''),
    (r'\^\{″\}',r'″'),
  	(r'\^\\prime',r'\''),
  	(r'\^\'',r'\''),
  	(r'\^\{。\}',r'\\circ'),
    
    # 删除单个关系运算符外括号
    (r'\{([-=><]|\+|\\le|\\ge|\.|\[|\]|\||\'|//|\*)\}', r'\1'),
    
    # 删除距离调整符号
    (r'\\\\', r'\n'),
    (r'\\!|\\,|\\;|\\:|\\quad|\\qquad|\\\s|\&\s', r''),
    
    # 其他符号替换
    (r'\\overset\s*\{\s*\\acute\s*\{\s*\}\s*\}\s*\{\s*\}\s*',r'\''),
    (r'\\acute\s*\{(.*)\}', r'\1'),
    (r'\\coprod', r'II'),
    (r'\\ell', r'l'),
    (r'\\overset\s*\\frown\s*\{(.*?)\}', r'\\overparen{\1}'),
    (r'\\grave\s*\{\s*\\\s*\}', r''),
    (r'\\grave\s*\{(.*?)\}', r'\1'),
    (r'\\leftrightharpoons', r'\,'),
    (r'\\mid', r'\|'),
    (r'\\perp', r'\\bot'),
    (r'\\propto', r'\\infty'),
    (r'\\star\s*|∗', r'\*'),
    (r'\\subsetneqq', r'\\subsetneq'),
    (r'\\supsetneqq', r'\\supsetneq'),
    (r'\\surd\s*([0-9]+)', r'\\sqrt{\1}'),
    (r'\\tfrac', r'\\frac'),
    (r'\\thickapprox', r'\\approx'),
    (r'\\widetilde', r'\\tilde'),
    (r'\\tilde\s*\{\s*\\\s*\}', r'\\sim'),
    
    # 修正数列
    (r'\\\s*\{\s*\+(.+?)\+\\\s*\}',r'{\1}'),
    (r'\\\s*\{\s*\\\s*\{(\s*.+?)\+\\\}\\\}',r'{\1}'),
    
    # 角度相关修正
    (r'([\u4e00-\u9fa5])\\circ', r'\1度'),
    (r'\^\{\s*\\circ\s*\}|\^\{∘}|\^o|\^\{o\}|º|∘|˚|\^\s*\\circ', r'\\circ'),
    
    # 修正 BEGIN END 环境
    (r'array|Bmatrix|bmatrix|pmatrix|smallmatrix|vmatrix', r'matrix'),
    (r'\s*\\begin\s*\{matrix\}\{l+\}\s*', r''),
    (r'\s*\\begin\s*\{matrix\}\{.\}\s*', r''),
    (r'\s*\\begin\s*\{matrix\}\s*', r''),
    (r'\s*\\end\s*\{matrix\}\s*', r''),
    (r'\s*\\begin\s*\{align\}\{l+\}\s*', r''),
    (r'\s*\\begin\s*\{align\}\{.\}\s*', r''),
    (r'\s*\\begin\s*\{align\}\s*', r''),
    (r'\s*\\end\s*\{align\}\s*', r''),
    (r'\s*\\begin\s*\{gathered\}\s*', r''),
    (r'\s*\\end\s*\{gathered\}\s*', r''),
    (r'\s*\\begin\s*\{cases\}\s*', r''),
    (r'\s*\\end\s*\{cases\}\s*', r''),

    # 修正上下角标
    (r'\{(\d+),\}', r'{\1},'),
    (r'(\^|_)\{\s*([0-9]+)\s*\}', r'\1{\2}'),
    (r'\\frac\s*\{([0-9a-zA-Z]+)\}\s*\{([0-9a-zA-Z]+)\}_\{([0-9a-zA-Z]+)\}', r'\\frac{\1}{{\2}_{\3}}'),
    (r'\\frac\s*\{([0-9a-zA-Z]+)\}\s*\{([0-9a-zA-Z]+)\}\^\{([0-9a-zA-Z]+)\}', r'(\\frac{\1}{\2})^{\3}'),
    (r'\\frac\s*\{([0-9a-zA-Z\\\s]+)\}_\{([0-9a-zA-Z\\\s]+)\}\s*\{([0-9a-zA-Z\\\s]+)\}_\{([0-9a-zA-Z\\\s]+)\}', r'\\frac{{\1}_{\2}}{{\3}_{\4}}'),
    (r'\\frac\s*\{([0-9a-zA-Z\\\s]+)\}_\{([0-9a-zA-Z\\\s]+)\}\s*\{', r'\\frac{{\1}_{\2}}{'),
    (r'\\frac\s*\{([0-9a-zA-Z]+)\}\^\{([0-9a-zA-Z]+)\}\s*\{', r'\\frac{{\1}^{\2}}{'),
    (r'([^_])\{(.)\}\^', r'\1\2^'),
    (r'\{(.)\}_([^_])', r'\1_\2'),
    (r'_\{([aehijklmnoprstuvx0-9\+\-\(\)])([aehijklmnoprstuvx0-9\+\-\(\)]+)\}', r'_{\1}_{\2}'),
    (r'\^\{([a-pr-z0-9\+\-\(\)])([a-pr-z0-9\+\-\(\)]+)\}', r'^{\1}^{\2}'),


    # 修正弧
    (r'\\overparen\s*\{([a-zA-Z]+)\}', r'弧\1'),

    # 修正无限循环
    (r'\\overset\.\{(\d)\}', r'\1\1\1\1\1…'),
    (r'\\overset\{\s*.+?\s*\}\{(\d)\}', r'\1\1\1\1\1…'),

    # 修正平均数
    (r'\\overline\s*\{x\}', r'x̄'),

    # 处理简单开根号
    (r'\s*\\sqrt\s*\{\s*([0-9a-zA-Z]+)\s*\}', r'√\1'),

    # 处理简单分式
    (r'dfrac', r'frac'),
    (r'\\frac\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉]+)\s*\}\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉]+)\s*\}', r' \1/\2 '),
    (r'\\frac\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉⁰¹²³⁴⁵⁶⁷⁸⁹√\+\-\(\)]+)\s*\}\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉]+)\s*\}', r' (\1)/\2 '),
    (r'\\frac\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉]+)\s*\}\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉⁰¹²³⁴⁵⁶⁷⁸⁹√\+\-\(\)]+)\s*\}', r' \1/(\2) '),
    (r'\\frac\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉⁰¹²³⁴⁵⁶⁷⁸⁹√\+\-\(\)]+)\s*\}\s*\{\s*([0-9a-zA-Z₀₁₂₃₄₅₆₇₈₉⁰¹²³⁴⁵⁶⁷⁸⁹√\+\-\(\)]+)\s*\}', r' (\1)/(\2) '),
    (r'\\frac\s*\{(.*?)\}\s*\{(.*?)\}', r' (\1)/(\2) '),
    # 处理开根号
    (r'\s*\\sqrt\s*\{([0-9a-zA-Z\\]+)\}', r'√\1'),
    (r'\s*\\sqrt\s*\{(.+?)\}', r'√(\1) '),

    # 处理无法处理的根号
    (r'\\sqrt', r'√'),

    # 处理平行
    (r'∥', r'//'),

    # 修正角度
    (r'\^\{\s*˚\s*\}', r'˚'),
    (r'\{(\d+)\}˚', r'\1˚'),

    # 修正面积
    (r'S\s*_\{(.+?)\}', r'S[\1]'),
    (r'C\s*_\{(.+?)\}', r'C[\1]'),

    # 调整括号嵌套
    (r'\+(\}|\]|\))', r'\1'),
    (r'(\{|\[|\()\+', r'\1'),
    (r'\+,', r','),
    (r'\|\{(.+?)\}\|',r'|\1|'),
    (r'\(\{(.+?)\}\)',r'(\1)'),
    (r'\[\{(.+?)\}\]',r'[\1]'),
    (r'\{\{(.+?)\}\}',r'{\1}'),
    (r'\{\[(.+?)\]\}',r'{\1}'),
    (r'\{\((.+?)\)\}',r'(\1)'),
    (r'\(\((.+?)\)\)',r'(\1)'),

    (r'\{\s*\}',r''),
    (r'\{\s*([=≠≅\|\-\+±×÷∠Δ⊙,;.，；。π]+)\s*\}',r'\1'),

    # ^{a}^{b} -> ^{ab}
    ('\^\{([^\{\}]*?)\}\s*\^\{([^\{\}]*?)\}', '^{\\1\\2}'),
    (r'\\text\{(.*?)\}',r'\1'),
]

re_compiled_pair = [
    [re.compile(e[0]), e[1]] for e in re_replace_pair
]


unicode_replace_pair = [
    ('\\#', '#'), 
    ('\\$', '$'), 
    ('\\%', '%'), 
    ('\\&', '&'), 
    ('\\:', '\u205f'), 
    ('\\=','='),
    ('\\AC', '∿'), 
    ('\\APLcomment', '⍝'), 
    ('\\APLdownarrowbox', '⍗'), 
    ('\\APLinput', '⍞'), 
    ('\\APLinv', '⌹'), 
    ('\\APLleftarrowbox', '⍇'), 
    ('\\APLlog', '⍟'), 
    ('\\APLrightarrowbox', '⍈'), 
    ('\\APLuparrowbox', '⍐'), 
    ('\\Alpha', 'A'),
    ('\\Angstroem', 'Å'), 
    ('\\Beta', 'B'),
    ('\\Bot', '⫫'), 
    ('\\Box', '□'),
    ('\\Bumpeq', '≎'), 
    ('\\CIRCLE', '●'), 
    ('\\Cap', '⋒'), 
    ('\\CapitalDifferentialD', 'ⅅ'), 
    ('\\CheckedBox', '☑'), 
    ('\\Chi', 'Χ'),
    ('\\Circle', '○'), 
    ('\\Coloneqq', '⩴'), 
    ('\\ComplexI', 'ⅈ'), 
    ('\\ComplexJ', 'ⅉ'), 
    ('\\Cup', '⋓'), 
    ('\\Delta', 'Δ'),
    ('\\Diamond', '◇'), 
    ('\\Diamondblack', '◆'), 
    ('\\Diamonddot', '⟐'), 
    ('\\DifferentialD', 'ⅆ'), 
    ('\\Digamma', 'Ϝ'), 
    ('\\Doteq', '≑'), 
    ('\\DownArrowBa', '⤓'), 
    ('\\DownLeftTeeVecto', '⥞'), 
    ('\\DownLeftVectorBa', '⥖'), 
    ('\\DownRightTeeVecto', '⥟'), 
    ('\\DownRightVectorBa', '⥗'), 
    ('\\Downarrow', '⇓'),
    ('\\Epsilon', 'E'),
    ('\\Equal', '⩵'), 
    ('\\Eta', 'H'),
    ('\\Eule', 'ℇ'), 
    ('\\ExponetialE', 'ⅇ'), 
    ('\\Finv', 'Ⅎ'), 
    ('\\Gamma', 'Γ'),
    ('\\Im', 'ℑ'),
    ('\\Iota', 'I'),
    ('\\Join', '⨝'), 
    ('\\Kappa', 'K'),
    ('\\Koppa', 'Ϟ'), 
    ('\\LEFTCIRCLE', '◖'), 
    ('\\LEFTcircle', '◐'), 
    ('\\LHD', '◀'), 
    ('\\LVec', 'x⃖'), 
    ('\\Lambda', 'Λ'),
    ('\\Lbag', '⟅'), 
    ('\\LeftArrowBa', '⇤'), 
    ('\\LeftDownTeeVecto', '⥡'), 
    ('\\LeftDownVectorBa', '⥙'), 
    ('\\LeftTeeVecto', '⥚'), 
    ('\\LeftTriangleBa', '⧏'), 
    ('\\LeftUpTeeVecto', '⥠'), 
    ('\\LeftUpVectorBa', '⥘'), 
    ('\\LeftVectorBa', '⥒'), 
    ('\\Leftarrow', '⇐'),
    ('\\Leftrightarrow', '⇔'),
    ('\\Lleftarrow', '⇚'), 
    ('\\Longleftarrow', '⟸'), 
    ('\\Longleftrightarrow', '⟺'), 
    ('\\Longmapsfrom', '⟽'), 
    ('\\Longmapsto', '⟾'), 
    ('\\Longrightarrow', '⟹'), 
    ('\\Lparen', '⦅'), 
    ('\\Lsh', '↰'), 
    ('\\MapsDown', '↧'), 
    ('\\MapsUp', '↥'), 
    ('\\Mapsfrom', '⤆'), 
    ('\\Mapsto', '⤇'), 
    ('\\Micro', 'µ'), 
    ('\\Mu', 'M'),
    ('\\Nearrow', '⇗'), 
    ('\\NestedGreaterGreate', '⪢'), 
    ('\\NestedLessLess', '⪡'), 
    ('\\NotGreaterLess', '≹'), 
    ('\\NotGreaterTilde', '≵'), 
    ('\\NotLessTilde', '≴'), 
    ('\\Nu', 'N'),
    ('\\Nwarrow', '⇖'), 
    ('\\Omega', 'Ω'),
    ('\\Omicron', 'O'),
    ('\\Phi', 'Φ'),
    ('\\Pi', 'Π'),
    ('\\Proportion', '∷'), 
    ('\\Psi', 'Ψ'),
    ('\\Qoppa', 'Ϙ'), 
    ('\\RHD', '▶'), 
    ('\\RIGHTCIRCLE', '◗'), 
    ('\\RIGHTcircle', '◑'), 
    ('\\Rbag', '⟆'), 
    ('\\Re', 'ℜ'),
    ('\\Rho', 'P'),
    ('\\RightArrowBa', '⇥'), 
    ('\\RightDownTeeVecto', '⥝'), 
    ('\\RightDownVectorBa', '⥕'), 
    ('\\RightTeeVecto', '⥛'), 
    ('\\RightTriangleBa', '⧐'), 
    ('\\RightUpTeeVecto', '⥜'), 
    ('\\RightUpVectorBa', '⥔'), 
    ('\\RightVectorBa', '⥓'), 
    ('\\Rightarrow', '⇒'),
    ('\\Rparen', '⦆'), 
    ('\\Rrightarrow', '⇛'), 
    ('\\Rsh', '↱'), 
    ('\\S', '§'), 
    ('\\Same', '⩶'), 
    ('\\Sampi', 'Ϡ'), 
    ('\\Searrow', '⇘'), 
    ('\\Sigma', 'Σ'),
    ('\\Square', '☐'), 
    ('\\Stigma', 'Ϛ'), 
    ('\\Subset', '⋐'), 
    ('\\Sun', '☉'), 
    ('\\Supset', '⋑'), 
    ('\\Swarrow', '⇙'), 
    ('\\Tau', 'T'),
    ('\\Theta', 'Θ'),
    ('\\Top', '⫪'), 
    ('\\UpArrowBa', '⤒'), 
    ('\\Uparrow', '⇑'),
    ('\\Updownarrow', '⇕'), 
    ('\\Upsilon', 'Y'),
    ('\\VDash', '⊫'), 
    ('\\VERT', '⦀'), 
    ('\\Vdash', '⊩'), 
    ('\\Vvdash', '⊪'), 
    ('\\XBox', '☒'), 
    ('\\Xi', 'Ξ'),
    ('\\Yup', '⅄'), 
    ('\\Zeta', 'Z'),
    ('\\\\beginsplit', ''),
    ('\\\{', '\{'), 
    ('\\\|', '‖'),
    ('\\\}', '\}'), 
    ('\\_', '_'), 
    ('\\acute', ' ́'), 
    ('\\aleph', 'ℵ'),
    ('\\allequal', '≌'),
    ('\\alpha', 'α'),
    ('\\amalg', '⨿'), 
    ('\\ancho', '⚓'), 
    ('\\angle', '∠'),
    ('\\approx', '≈'),
    ('\\approxeq', '≊'),
    ('\\approxnotequal', '≆'),
    ('\\aquarius', '♒'), 
    ('\\aries', '♈'), 
    ('\\arrowbullet', '➢'), 
    ('\\ast', '∗'), 
    ('\\asymp', '≍'),
    ('\\backepsilon', '϶'), 
    ('\\backprime', '‵'), 
    ('\\backsim', '∽'),
    ('\\backsimeq', '⋍'), 
    ('\\backslash', '\\'), 
    ('\\ballotx', '✗'), 
    ('\\ba', ' ̄'), 
    ('\\barin', '⋶'), 
    ('\\barleftharpoon', '⥫'), 
    ('\\barrightharpoon', '⥭'), 
    ('\\barwedge', '⊼'), 
    ('\\because', '∵'), 
    ('\\beginsplit', ''),
    ('\\beta', 'β'),
    ('\\beth', 'ℶ'),
    ('\\between', '≬'), 
    ('\\bigcap', '⋂'),
    ('\\bigcirc', '◯'),
    ('\\bigcup', '⋃'),
    ('\\biginterleave', '⫼'), 
    ('\\bigodot', '⨀'), 
    ('\\bigoplus', '⨁'), 
    ('\\bigotimes', '⨂'), 
    ('\\bigsqcap', '⨅'), 
    ('\\bigsqcup', '⨆'), 
    ('\\bigsta', '★'),
    ('\\bigtriangledown', '▽'), 
    ('\\bigtriangleup', '△'), 
    ('\\biguplus', '⨄'), 
    ('\\bigvee', '⋁'), 
    ('\\bigwedge', '⋀'), 
    ('\\bij', '⤖'), 
    ('\\biohazard', '☣'), 
    ('\\blacklozenge', '⧫'), 
    ('\\blacksmiley', '☻'), 
    ('\\blacksquare', '◼'), 
    ('\\blacktriangledown', '▾'), 
    ('\\blacktriangleleft', '◂'), 
    ('\\blacktriangleright', '▸'), 
    ('\\blacktriangleup', '▴'), 
    ('\\bot', '⊥'),
    ('\\bowtie', '⋈'), 
    ('\\boxast', '⧆'), 
    ('\\boxba', '◫'), 
    ('\\boxbox', '⧈'), 
    ('\\boxbslash', '⧅'), 
    ('\\boxcircle', '⧇'), 
    ('\\boxdot', '⊡'), 
    ('\\boxminus', '⊟'), 
    ('\\boxplus', '⊞'), 
    ('\\boxslash', '⧄'), 
    ('\\boxtimes', '⊠'), 
    ('\\breve', ' ̆'), 
    ('\\bullet', '•'), 
    ('\\bumpeq', '≏'), 
    ('\\cance', '♋'), 
    ('\\cap', '∩'),
    ('\\capricornus', '♑'), 
    ('\\cat', '⁀'), 
    ('\\cdot', '⋅'),
    ('\\cdots', '…'),
    ('\\cent', '¢'), 
    ('\\check', ' ̌'), 
    ('\\checkmark', '✓'), 
    ('\\chi', 'χ'),
    ('\\circ', '˚'),
    ('\\circeq', '≗'), 
    ('\\circlearrowleft', '↺'), 
    ('\\circlearrowright', '↻'), 
    ('\\circled', '®'), 
    ('\\circledast', '⊛'), 
    ('\\circledbslash', '⦸'), 
    ('\\circledcirc', '⊚'), 
    ('\\circleddash', '⊝'), 
    ('\\circledgt', '⧁'), 
    ('\\circledless', '⧀'), 
    ('\\clubsuit', '♣'), 
    ('\\coloneq', '≔'), 
    ('\\complement', '∁'),
    ('\\cong', '≅'),
    ('\\coprod', '∐'),
    ('\\corresponds', '≙'), 
    ('\\cup', '∪'),
    ('\\curlyeqprec', '⋞'), 
    ('\\curlyeqsucc', '⋟'), 
    ('\\curlyvee', '⋎'), 
    ('\\curlywedge', '⋏'), 
    ('\\curvearrowleft', '↶'), 
    ('\\curvearrowright', '↷'), 
    ('\\dagge', '†'), 
    ('\\daleth', 'ℸ'), 
    ('\\dashleftarrow', '⇠'), 
    ('\\dashrightarrow', '⇢'), 
    ('\\dashv', '⊣'), 
    ('\\ddagge', '‡'), 
    ('\\ddddot', 'x⃜'), 
    ('\\dddot', 'x⃛'), 
    ('\\ddot', ' ̈'), 
    ('\\ddots', '⋱'), 
    ('\\delta', 'δ'),
    ('\\diamete', '⌀'), 
    ('\\diamond', '⋄'), 
    ('\\diamondsuit', '♢'), 
    ('\\digamma', 'ϝ'), 
    ('\\div', '÷'),
    ('\\divideontimes', '⋇'), 
    ('\\dlsh', '↲'), 
    ('\\dot', ' ̇'), 
    ('\\doteq', '≐'),
    ('\\doteqdot', '≑'),
    ('\\dotplus', '∔'),
    ('\\dots', '…'),
    ('\\doublebarwedge', '⩞'), 
    ('\\downarrow', '↓'),
    ('\\downdownarrows', '⇊'), 
    ('\\downdownharpoons', '⥥'), 
    ('\\downharpoonleft', '⇃'), 
    ('\\downharpoonright', '⇂'), 
    ('\\downuparrows', '⇵'), 
    ('\\downupharpoons', '⥯'), 
    ('\\drsh', '↳'), 
    ('\\dsub', '⩤'), 
    ('\\earth', '♁'), 
    ('\\eighthnote', '♪'), 
    ('\\ell', 'ℓ'), 
    ('\\emptyset', '∅'),
    ('\\endsplit', ''),
    ('\\ente','\n'),
    ('\\epsilon', 'ϵ'),
    ('\\eqcirc', '≖'), 
    ('\\eqcolon', '∹'), 
    ('\\eqsim', '≂'), 
    ('\\eqslantgt', '⪖'), 
    ('\\eqslantless', '⪕'), 
    ('\\equiv', '≡'),
    ('\\eta', 'η'),
    ('\\eth', 'ð'), 
    ('\\exists', '∃'),
    ('\\fallingdotseq', '≒'), 
    ('\\fcmp', '⨾'), 
    ('\\female', '♀'), 
    ('\\ffun', '⇻'), 
    ('\\finj', '⤕'), 
    ('\\fint', '⨏'), 
    ('\\flat', '♭'),
    ('\\forall', '∀'),
    ('\\fourth', '⁗'), 
    ('\\frown', '⌢'), 
    ('\\frownie', '☹'), 
    ('\\gamma', 'γ'),
    ('\\ge', '≥'),  
    ('\\gemini', '♊'), 
    ('\\geq', '≥'), 
    ('\\geqq', '≥'),    
    ('\\geqslant', '⩾'),
    ('\\gg', '≫'),
    ('\\ggcurly', '⪼'), 
    ('\\ggg', '⋙'), 
    ('\\gimel', 'ℷ'), 
    ('\\gnapprox', '⪊'), 
    ('\\gneq', '⪈'),
    ('\\gneqq', '≩'),
    ('\\gnsim', '⋧'), 
    ('\\grave', ' ̀'), 
    ('\\greaterequivlnt', '≳'),
    ('\\gtrapprox', '⪆'), 
    ('\\gtrdot', '⋗'), 
    ('\\gtreqless', '⋛'), 
    ('\\gtreqqless', '⪌'), 
    ('\\gtrless', '≷'), 
    ('\\gtrsim', '≳'), 
    ('\\hash', '⋕'), 
    ('\\hat', ' ̂'), 
    ('\\heartsuit', '♡'), 
    ('\\hline','\n——————\n'),
    ('\\hookleftarrow', '↩'),
    ('\\hookrightarrow', '↪'),
    ('\\hslash', 'ℏ'), 
    ('\\iddots', '⋰'), 
    ('\\iiiint', '⨌'),
    ('\\iiint', '∭'),
    ('\\iint', '∬'),
    ('\\imath', 'ı'), 
    ('\\in', '∈'),
    ('\\infty', '∞'),
    ('\\int', '∫'),
    ('\\intercal', '⊺'), 
    ('\\interleave', '⫴'), 
    ('\\invamp', '⅋'), 
    ('\\invdiamete', '⍉'), 
    ('\\invneg', '⌐'), 
    ('\\iota', 'ι'),
    ('\\jmath', 'ȷ'), 
    ('\\jupite', '♃'), 
    ('\\kappa', 'κ'),
    ('\\koppa', 'ϟ'), 
    ('\\lambda', 'λ'),
    ('\\land', '∧'),
    ('\\lang', '⟪'), 
    ('\\langle', '<'),
    ('\\lazysinv', '∾'),
    ('\\lblot', '⦉'), 
    ('\\lbrack', '['),
    ('\\lceil', '⌈'),
    ('\\ldots', '…'),
    ('\\le', '≤'),
    ('\\leadsto', '⤳'), 
    ('\\leftarrow', '←'),
    ('\\leftarrowtail', '↢'),
    ('\\leftarrowtriangle', '⇽'), 
    ('\\leftbarharpoon', '⥪'), 
    ('\\leftharpoondown', '↽'), 
    ('\\leftharpoonup', '↼'), 
    ('\\leftleftarrows', '⇇'), 
    ('\\leftleftharpoons', '⥢'), 
    ('\\leftmoon', '☾'), 
    ('\\leftrightarrow', '↔'),
    ('\\leftrightarrows', '⇆'), 
    ('\\leftrightarrowtriangle', '⇿'), 
    ('\\leftrightharpoon', '⥊'), 
    ('\\leftrightharpoondown', '⥐'), 
    ('\\leftrightharpoons', '⇋'),
    ('\\leftrightharpoonup', '⥎'), 
    ('\\leftrightsquigarrow', '↭'), 
    ('\\leftslice', '⪦'), 
    ('\\leftsquigarrow', '⇜'), 
    ('\\leftthreetimes', '⋋'), 
    ('\\leftupdownharpoon', '⥑'), 
    ('\\leo', '♌'), 
    ('\\leq', '≤'),
    ('\\leqq', '≤'),
    ('\\leqslant', '⩽'),
    ('\\lessapprox', '⪅'), 
    ('\\lessdot', '⋖'), 
    ('\\lesseqgt', '⋚'), 
    ('\\lesseqqgt', '⪋'), 
    ('\\lessequivlnt', '≲'),
    ('\\lessgt', '≶'), 
    ('\\lesssim', '≲'), 
    ('\\lfloo', '⌊'),
    ('\\lgroup', '⟮'), 
    ('\\lhd', '◁'), 
    ('\\libra', '♎'), 
    ('\\lightning', '↯'), 
    ('\\limg', '⦇'), 
    ('\\ll', '≪'),
    ('\\llbracket', '⟦'), 
    ('\\llcorne', '⌞'), 
    ('\\llcurly', '⪻'), 
    ('\\lll', '⋘'), 
    ('\\lnapprox', '⪉'), 
    ('\\lneq', '⪇'),
    ('\\lneqq', '≨'),
    ('\\lnsim', '⋦'), 
    ('\\longleftarrow', '⟵'), 
    ('\\longleftrightarrow', '⟷'), 
    ('\\longmapsfrom', '⟻'), 
    ('\\longmapsto', '⟼'), 
    ('\\longrightarrow', '⟶'), 
    ('\\looparrowleft', '↫'), 
    ('\\looparrowright', '↬'), 
    ('\\lo', '∨'),
    ('\\lozenge', '◊'), 
    ('\\lrcorne', '⌟'), 
    ('\\ltimes', '⋉'), 
    ('\\lvec', 'x⃐'), 
    ('\\male', '♂'), 
    ('\\maltese', '✠'), 
    ('\\mapsfrom', '↤'), 
    ('\\mapsto', '↦'),
    ('\\measuredangle', '∡'),
    ('\\medbullet', '⚫'), 
    ('\\medcirc', '⚪'), 
    ('\\mercury', '☿'), 
    ('\\mho', '℧'), 
    ('\\mid', '∣'),
    ('\\models', '⊧'), 
    ('\\mp', '∓'),
    ('\\mu', 'μ'),
    ('\\multimap', '⊸'), 
    ('\\multimapboth', '⧟'), 
    ('\\multimapdotbothA', '⊶'), 
    ('\\multimapdotbothB', '⊷'), 
    ('\\multimapinv', '⟜'), 
    ('\\nLeftarrow', '⇍'), 
    ('\\nLeftrightarrow', '⇎'), 
    ('\\nRightarrow', '⇏'), 
    ('\\nVDash', '⊯'), 
    ('\\nVdash', '⊮'), 
    ('\\nabla', '∇'),
    ('\\napprox', '≉'), 
    ('\\natural', '♮'),
    ('\\ncong', '≇'), 
    ('\\ne', '≠'),    
    ('\\nearrow', '↗'),
    ('\\neg', '¬'),
    ('\\neptune', '♆'), 
    ('\\neq', '≠'),
    ('\\nequiv', '≢'), 
    ('\\nexists', '∄'),
    ('\\ngeq', '≱'),
    ('\\ngt', '≯'),
    ('\\ni', '∋'),
    ('\\nleftarrow', '↚'), 
    ('\\nleftrightarrow', '↮'), 
    ('\\nleq', '≰'),
    ('\\nless', '≮'),
    ('\\nmid', '∤'),
    ('\\nni', '∌'), 
    ('\\not', ' ̸'), 
    ('\\not\\approx', '≉'),
    ('\\not\\cong', '≇'),
    ('\\not\\equiv', '≢'),
    ('\\not\\exists', '∄'),
    ('\\not\\in', '∉'),
    ('\\not\\mid', '∤'),
    ('\\not\\ni', '∌'),
    ('\\not\\parallel', '∦'),
    ('\\not\\sim', '≁'),
    ('\\not\\simeq', '≄'),
    ('\\not\\subset', '⊄'),
    ('\\not\\subseteq', '⊈'),
    ('\\not\\supset', '⊅'),
    ('\\not\\supseteq', '⊉'),
    ('\\notasymp', '≭'), 
    ('\\notbackslash', '⍀'), 
    ('\\notin', '∉'),
    ('\\notslash', '⌿'), 
    ('\\nparallel', '∦'),
    ('\\nprec', '⊀'),
    ('\\npreceq', '⋠'), 
    ('\\nrightarrow', '↛'), 
    ('\\nsim', '≁'), 
    ('\\nsimeq', '≄'), 
    ('\\nsqsubseteq', '⋢'), 
    ('\\nsqsupseteq', '⋣'), 
    ('\\nsubset', '⊄'), 
    ('\\nsubseteq', '⊈'), 
    ('\\nsucc', '⊁'),
    ('\\nsucceq', '⋡'), 
    ('\\nsupset', '⊅'), 
    ('\\nsupseteq', '⊉'), 
    ('\\ntriangleleft', '⋪'), 
    ('\\ntrianglelefteq', '⋬'), 
    ('\\ntriangleright', '⋫'), 
    ('\\ntrianglerighteq', '⋭'), 
    ('\\nu', 'ν'),
    ('\\nvDash', '⊭'), 
    ('\\nvdash', '⊬'), 
    ('\\nwarrow', '↖'),
    ('\\odot', '⊙'),
    ('\\oiiint', '∰'), 
    ('\\oiint', '∯'), 
    ('\\oint', '∮'),
    ('\\ointctrclockwise', '∳'), 
    ('\\omega', 'ω'),
    ('\\omicron', 'ο'),
    ('\\ominus', '⊖'),
    ('\\oplus', '⊕'),
    ('\\oslash', '⊘'),
    ('\\otimes', '⊗'),
    ('\\overbrace', '⏞'), 
    ('\\overleftrightarrow', 'x⃡'), 
    ('\\overline', ' ̅'), 
    ('\\overparen', '⏜'), 
    ('\\parallel', '∥'),
    ('\\partial', '∂'),
    ('\\pencil', '✎'), 
    ('\\perp', '⟂'), 
    ('\\pfun', '⇸'), 
    ('\\phi', 'ϕ'),
    ('\\pi', 'π'),
    ('\\pinj', '⤔'), 
    ('\\pisces', '♓'), 
    ('\\pitchfork', '⋔'), 
    ('\\pluto', '♇'), 
    ('\\pm', '±'),
    ('\\pointright', '☞'), 
    ('\\pounds', '£'), 
    ('\\prec', '≺'),
    ('\\precapprox', '≾'),
    ('\\preccurlyeq', '≼'),
    ('\\preceq', '⪯'),
    ('\\preceqq', '⪳'), 
    ('\\precnapprox', '⪹'), 
    ('\\precneqq', '⪵'),
    ('\\precnsim', '⋨'), 
    ('\\precsim', '≾'), 
    ('\\prime', '′'), 
    ('\\prod', '∏'),
    ('\\propto', '∝'),
    ('\\psi', 'ψ'),
    ('\\psu', '⤀'), 
    ('\\qoppa', 'ϙ'), 
    ('\\quad', '\u2001'), 
    ('\\quarternote', '♩'), 
    ('\\radiation', '☢'), 
    ('\\rang', '⟫'), 
    ('\\rangle', '>'),
    ('\\rblot', '⦊'), 
    ('\\rbrack', '\]'), 
    ('\\rceil', '⌉'),
    ('\\recycle', '♻'), 
    ('\\rfloo', '⌋'),
    ('\\rgroup', '⟯'), 
    ('\\rhd', '▷'), 
    ('\\rho', 'ρ'),
    ('\\rightangle', '∟'), 
    ('\\rightarrow', '→'),
    ('\\rightarrowtail', '↣'),
    ('\\rightarrowtriangle', '⇾'), 
    ('\\rightbarharpoon', '⥬'), 
    ('\\rightharpoondown', '⇁'),
    ('\\rightharpoonup', '⇀'),
    ('\\rightleftarrows', '⇄'), 
    ('\\rightleftharpoon', '⥋'), 
    ('\\rightleftharpoons', '⇌'),
    ('\\rightmoon', '☽'), 
    ('\\rightrightarrows', '⇉'), 
    ('\\rightrightharpoons', '⥤'), 
    ('\\rightslice', '⪧'), 
    ('\\rightsquigarrow', '⇝'),
    ('\\rightthreetimes', '⋌'), 
    ('\\rightupdownharpoon', '⥏'), 
    ('\\rimg', '⦈'), 
    ('\\risingdotseq', '≓'), 
    ('\\rrbracket', '⟧'), 
    ('\\rsub', '⩥'), 
    ('\\rtimes', '⋊'), 
    ('\\sagittarius', '♐'), 
    ('\\sampi', 'ϡ'), 
    ('\\saturn', '♄'), 
    ('\\scorpio', '♏'), 
    ('\\searrow', '↘'),
    ('\\second', '″'), 
    ('\\setminus', '∖'),
    ('\\sharp', '♯'),
    ('\\sigma', 'σ'),
    ('\\sim', '∼'),
    ('\\simeq', '≃'),
    ('\\sixteenthnote', '♬'), 
    ('\\skull', '☠'), 
    ('\\slash', '∕'), 
    ('\\smallsetminus', '∖'), 
    ('\\smalltriangledown', '▿'), 
    ('\\smalltriangleleft', '◃'), 
    ('\\smalltriangleright', '▹'), 
    ('\\smalltriangleup', '▵'), 
    ('\\smea',''),
    ('\\smile', '⌣'), 
    ('\\smiley', '☺'), 
    ('\\space',''),
    ('\\spadesuit', '♠'), 
    ('\\spddot', '¨'), 
    ('\\sphat', ''), 
    ('\\sphericalangle', '∢'),
    ('\\spot', '⦁'), 
    ('\\sptilde', '~'), 
    ('\\sqcap', '⊓'), 
    ('\\sqcup', '⊔'), 
    ('\\sqint', '⨖'), 
    ('\\sqrt', '√'), 
    ('\\sqrt\[3\]', '∛'), 
    ('\\sqrt\[4\]', '∜'), 
    ('\\sqsubset', '⊏'), 
    ('\\sqsubseteq', '⊑'), 
    ('\\sqsupset', '⊐'), 
    ('\\sqsupseteq', '⊒'), 
    ('\\square', '☐'),
    ('\\sslash', '⫽'), 
    ('\\sta', '⋆'), 
    ('\\steaming', '☕'), 
    ('\\stigma', 'ϛ'), 
    ('\\strictfi', '⥼'), 
    ('\\strictif', '⥽'), 
    ('\\subset', '⊂'),
    ('\\subseteq', '⊆'),
    ('\\subseteqq', '⫅'), 
    ('\\subsetneq', '⊊'),
    ('\\subsetneqq', '⫋'), 
    ('\\succ', '≻'),
    ('\\succapprox', '≿'),
    ('\\succcurlyeq', '≽'),
    ('\\succeq', '⪰'),
    ('\\succeqq', '⪴'), 
    ('\\succnapprox', '⪺'), 
    ('\\succneqq', '⪶'),
    ('\\succnsim', '⋩'), 
    ('\\succsim', '≿'), 
    ('\\sum', '∑'),
    ('\\sun', '☼'), 
    ('\\supset', '⊃'),
    ('\\supseteq', '⊇'),
    ('\\supseteqq', '⫆'), 
    ('\\supsetneq', '⊋'),
    ('\\supsetneqq', '⫌'), 
    ('\\surd', '√'),
    ('\\surfintegral', '∯'),
    ('\\swarrow', '↙'),
    ('\\swords', '⚔'), 
    ('\\talloblong', '⫾'), 
    ('\\tau', 'τ'),
    ('\\taurus', '♉'), 
    ('\\tcohm', 'Ω'), 
    ('\\textcelsius', '˚C'),
    ('\\textcircled\{10\}', '⑩'),
    ('\\textcircled\{11\}', '⑪'),
    ('\\textcircled\{12\}', '⑫'),
    ('\\textcircled\{13\}', '⑬'),
    ('\\textcircled\{14\}', '⑭'),
    ('\\textcircled\{15\}', '⑮'),
    ('\\textcircled\{1\}', '①'),
    ('\\textcircled\{2\}', '②'),
    ('\\textcircled\{3\}', '③'),
    ('\\textcircled\{4\}', '④'),
    ('\\textcircled\{5\}', '⑤'),
    ('\\textcircled\{6\}', '⑥'),
    ('\\textcircled\{7\}', '⑦'),
    ('\\textcircled\{8\}', '⑧'),
    ('\\textcircled\{9\}', '⑨'),
    ('\\therefore', '∴'), 
    ('\\theta', 'θ'),
    ('\\third', '‴'), 
    ('\\tilde', ' ̃'), 
    ('\\tildetrpl', '≋'),
    ('\\times', '×'),
    ('\\to', '→'),
    ('\\top', '⊤'),
    ('\\triangle', 'Δ'),
    ('\\trianglelefteq', '⊴'), 
    ('\\triangleq', '≜'), 
    ('\\trianglerighteq', '⊵'), 
    ('\\twoheadleftarrow', '↞'),
    ('\\twoheadrightarrow', '↠'),
    ('\\twonotes', '♫'), 
    ('\\ulcorne', '⌜'), 
    ('\\underba', ' ̱'), 
    ('\\underbrace', '⏟'), 
    ('\\underleftarrow', 'x⃮'), 
    ('\\underline', ' ̲'), 
    ('\\underparen', '⏝'), 
    ('\\underrightarrow', 'x⃯'), 
    ('\\uparrow', '↑'),
    ('\\updownarrow', '↕'), 
    ('\\updownarrows', '⇅'), 
    ('\\updownharpoons', '⥮'), 
    ('\\upharpoonleft', '↿'), 
    ('\\upharpoonright', '↾'), 
    ('\\uplus', '⊎'), 
    ('\\upsilon', 'υ'),
    ('\\upuparrows', '⇈'), 
    ('\\upupharpoons', '⥣'), 
    ('\\uranus', '♅'), 
    ('\\urcorne', '⌝'), 
    ('\\utilde', ' ̰'), 
    ('\\vDash', '⊨'), 
    ('\\varbeta', 'ϐ'), 
    ('\\varclubsuit', '♧'), 
    ('\\vardiamondsuit', '♦'), 
    ('\\varepsilon', 'ε'),
    ('\\varheartsuit', '♥'), 
    ('\\varkappa', 'ϰ'),
    ('\\varnothing', '∅'),
    ('\\varointclockwise', '∲'), 
    ('\\varphi', 'φ'),
    ('\\varpi', 'ϖ'),
    ('\\varprod', '⨉'), 
    ('\\varrho', 'ϱ'),
    ('\\varsigma', 'ς'),
    ('\\varspadesuit', '♤'), 
    ('\\vartheta', 'ϑ'),
    ('\\vartriangle', 'Δ'),
    ('\\vartriangleleft', '⊲'), 
    ('\\vartriangleright', '⊳'), 
    ('\\vdash', '⊢'),
    ('\\vdots', '⋮'), 
    ('\\vec', 'x⃑'), 
    ('\\vee', '∨'),
    ('\\veeba', '⊻'),
    ('\\virgo', '♍'), 
    ('\\volintegral', '∰'),
    ('\\warning', '⚠'), 
    ('\\wasylozenge', '⌑'), 
    ('\\wedge', '∧'),
    ('\\wp', '℘'),
    ('\\w', '≀'),
    ('\\xi', 'ξ'),
    ('\\yen', '¥'), 
    ('\\yinyang', '☯'), 
    ('\\zcmp', '⨟'), 
    ('\\zeta', 'ζ'),
    ('\\zhide', '⧹'), 
    ('\\zpipe', '⨠'), 
    ('\\zproject', '⨡'),  
]
    
unicode_re_pair = [     
    (r'\^\s*0', r'⁰'),
    (r'\^\s*1', r'¹'),
    (r'\^\s*2', r'²'),
    (r'\^\s*3', r'³'),
    (r'\^\s*4', r'⁴'),
    (r'\^\s*5', r'⁵'),
    (r'\^\s*6', r'⁶'),
    (r'\^\s*7', r'⁷'),
    (r'\^\s*8', r'⁸'),
    (r'\^\s*9', r'⁹'),
    (r'\^\s*\{0\}', r'⁰'),
    (r'\^\s*\{1\}', r'¹'),
    (r'\^\s*\{2\}', r'²'),
    (r'\^\s*\{3\}', r'³'),
    (r'\^\s*\{4\}', r'⁴'),
    (r'\^\s*\{5\}', r'⁵'),
    (r'\^\s*\{6\}', r'⁶'),
    (r'\^\s*\{7\}', r'⁷'),
    (r'\^\s*\{8\}', r'⁸'),
    (r'\^\s*\{9\}', r'⁹'),
    (r'\^\s*\{\(\}', r'⁽'),
    (r'\^\s*\{\)\}', r'⁾'),
    (r'\^\s*\{\+\}', r'⁺'),
    (r'\^\s*\{\-\}', r'⁻'),
    (r'\^\s*\{a\}', r'ᵃ'),
    (r'\^\s*\{b\}', r'ᵇ'),
    (r'\^\s*\{c\}', r'ᶜ'),
    (r'\^\s*\{d\}', r'ᵈ'),
    (r'\^\s*\{e\}', r'ᵉ'),
    (r'\^\s*\{f\}', r'ᶠ'),
    (r'\^\s*\{g\}', r'ᵍ'),
    (r'\^\s*\{h\}', r'ʰ'),
    (r'\^\s*\{i\}', r'ⁱ'),
    (r'\^\s*\{j\}', r'ʲ'),
    (r'\^\s*\{k\}', r'ᵏ'),
    (r'\^\s*\{l\}', r'ˡ'),
    (r'\^\s*\{m\}', r'ᵐ'),
    (r'\^\s*\{n\}', r'ⁿ'),
    (r'\^\s*\{o\}', r'ᵒ'),
    (r'\^\s*\{p\}', r'ᵖ'),
    (r'\^\s*\{r\}', r'ʳ'),
    (r'\^\s*\{s\}', r'ˢ'),
    (r'\^\s*\{t\}', r'ᵗ'),
    (r'\^\s*\{u\}', r'ᵘ'),
    (r'\^\s*\{v\}', r'ᵛ'),
    (r'\^\s*\{w\}', r'ʷ'),
    (r'\^\s*\{x\}', r'ˣ'),
    (r'\^\s*\{y\}', r'ʸ'),
    (r'\^\s*\{z\}', r'ᶻ'),
    (r'_\s*0', r'₀'),
    (r'_\s*1', r'₁'),
    (r'_\s*2', r'₂'),
    (r'_\s*3', r'₃'),
    (r'_\s*4', r'₄'),
    (r'_\s*5', r'₅'),
    (r'_\s*6', r'₆'),
    (r'_\s*7', r'₇'),
    (r'_\s*8', r'₈'),
    (r'_\s*9', r'₉'),
    (r'_\s*\{0\}', r'₀'),
    (r'_\s*\{1\}', r'₁'),
    (r'_\s*\{2\}', r'₂'),
    (r'_\s*\{3\}', r'₃'),
    (r'_\s*\{4\}', r'₄'),
    (r'_\s*\{5\}', r'₅'),
    (r'_\s*\{6\}', r'₆'),
    (r'_\s*\{7\}', r'₇'),
    (r'_\s*\{8\}', r'₈'),
    (r'_\s*\{9\}', r'₉'),
    (r'_\s*\{\(\}', r'₍'),
    (r'_\s*\{\)\}', r'₎'),
    (r'_\s*\{\+\}', r'₊'),
    (r'_\s*\{\-\}', r'₋'),
    (r'_\s*\{a\}', r'ₐ'),
    (r'_\s*\{e\}', r'ₑ'),
    (r'_\s*\{h\}', r'ₕ'),
    (r'_\s*\{i\}', r'ᵢ'),
    (r'_\s*\{j\}', r'ⱼ'),
    (r'_\s*\{k\}', r'ₖ'),
    (r'_\s*\{l\}', r'ₗ'),
    (r'_\s*\{m\}', r'ₘ'),
    (r'_\s*\{n\}', r'ₙ'),
    (r'_\s*\{o\}', r'ₒ'),
    (r'_\s*\{p\}', r'ₚ'),
    (r'_\s*\{r\}', r'ᵣ'),
    (r'_\s*\{s\}', r'ₛ'),
    (r'_\s*\{t\}', r'ₜ'),
    (r'_\s*\{u\}', r'ᵤ'),
    (r'_\s*\{v\}', r'ᵥ'),
    (r'_\s*\{x\}', r'ₓ'),
    (r'——————','='),
    (r'＝','='),
]

unicode_compiled_pair = [
    [re.compile(e[0]), e[1]] for e in unicode_re_pair
]


def remove_upprintable_chars(s):
    """移除所有不可见字符"""
    return ''.join(x for x in s if x.isprintable())
        
def preprocess(text):
    # 去除不可见字符
    text = remove_upprintable_chars(text)
    # unicode 标准化
    text = unicodedata.normalize('NFKC', text)
    # 上标解析
    text = re.sub(r'<sup.*?>(.*?)<\/sup.*?>', r'^\1', text)  # 上标符1
    text = re.sub(r'&sup(.*?)', r'^\1', text)  # 上标符2
    text = re.sub(r'<sub.*?>(.*?)<\/sub.*?>', r'\_\1', text)  # 下标符1
    text = re.sub(r'&sub(.*?)', r'\_\1', text)  # 下标符2
    # html解析
    text_byhtml = BeautifulSoup(text, features="lxml")
    # data-latex解码
    element = text_byhtml.find_all('img')
    for e in element:
        data_latex = e.get('data-latex')
        if data_latex:
            decoded_latex = urllib.parse.unquote(data_latex)
        else:
            decoded_latex = ' '
        e.replaceWith(decoded_latex) 
    
    raw_text = []
    for item in text_byhtml.stripped_strings:
        raw_text.append(item)
    text = ' '.join(raw_text)
    # 修正表达
    for pattern, repl in re_compiled_pair:
        text = re.sub(pattern, repl, text)
    #ifly-latex解析
    text = re.sub('ifly-latex-begin\s*(.*?)\s*ifly-latex-end', r'\1', text)
    text = re.sub('ifly_latex_begin\s*(.*?)\s*ifly_latex_end', r'\1', text)
    
    # 去除题目分值
    text = re.sub('\(.*?[0-9一二三四五六七八九十]{1,2}分.*?\)', '', text)
    # 去除题号
    text = re.sub('^([ ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ例ABCD【】]{0,4})([、\.．]+)', '', text)
    text = re.sub('^[\d]+\s*[\.．]\s*(.*)', '\\1', text)
    text = re.sub('^([一二三四五六七八九十 ]{0,4})([、\.．]+)', '', text)
    text = re.sub(r"\\textcircled\s*\{\s*\d+\s*\}", r'', text)
    text = re.sub(r"\\\\textcircled\s*\{\s*\d+\s*\}", r'', text)
    #选项删除
    text = re.sub(r"\s*[A-Z]\.\s*", r',', text)
    text = re.sub(r"\s*（\d+）\s*", r',', text)
    text = re.sub(r"\s*\(\d+\)\s*", r',', text)
    text = re.sub(r"[\.。,，:：!！？?;；]\s*,", r',', text)
    text = re.sub(r"^,", r'', text)
    #括号等
    text = re.sub(r"\(\s*\)", r'', text)
    text = re.sub(r"（\s*）", r'', text)
    text = re.sub(r"_+", r'', text)
    # 转unicode
    for pattern, repl in unicode_compiled_pair:
        text = re.sub(pattern, repl, text)
    for src, tgt in unicode_replace_pair:
        text = text.replace(src,tgt)
        
    text = re.sub(r"([^0-9])\s+", r'\1', text)
    text = re.sub(r"\s+([^0-9])", r'\1', text)
    text = re.sub(r"\s+", r" ", text)
    return text





if __name__ == '__main__':
    
    
    # file_02 = open('data_before_and_after_process_0203_0830.json', "r", encoding="utf-8")
    # for line in file_02:
    #     item = json.loads(line)
    #     # raw_content = item['raw_content']
    #     # processed_content = item['processed_content']
    #     raw_analysis = item['raw_analysis']
    #     processed_analysis = item['processed_analysis']
    #     custom_processed_analysis = preprocess(raw_analysis)
    
    #    with open('data_test_2024092516.txt','r',encoding='utf-8') as f:
    #         lines = f.readlines()
    #         for line in lines:
    #             text = preprocess(line)
    
    text = '\"<div><p x=\"12\" y=\"5\" width=\"1230\" height=\"46\" class=\"cut-check\" data-value=\"0\" style=\"\">7.如图,在Rtifly-latex-begin \\triangle ifly-latex-endABC中,ifly-latex-begin \\angle ifly-latex-endC=ifly-latex-begin 90^{ \\circ } ifly-latex-end,以顶点A为圆心,适当长为半径画弧,分别交AC,AB于点M,N,再分别</p><p x=\"41\" y=\"50\" width=\"1199\" height=\"64\" class=\"cut-check\" data-value=\"1\" style=\"\">以点M,N为圆心,大于ifly-latex-begin \\frac{1}{2} ifly-latex-endMN的长为半径画弧,两弧交于点P,作射线AP交边BC于点D,若CD=4,</p><p x=\"38\" y=\"115\" width=\"429\" height=\"40\" class=\"cut-check\" data-value=\"2\" style=\"\">AB=15,则ifly-latex-begin \\triangle ifly-latex-endABD的面积是<u>          </u>.</p></div>\"'
    text = preprocess(text)
    print(text)     
    
    # file_02 = open('/train13/bdr/jzwu4/data/pretrain_data/corpus_02.json', "r", encoding="utf-8")
    #file_02 = open('/work/aiexam/lfhao/tiku_process/tiku_data/raw_data/html_19w/split_data_01.json', "r", encoding="utf-8")
    # for line in file_02:
        
    #     item = json.loads(line)
    #     if random.random()>0.01:
    #         continue
    #     print(item['id'])
    #     print(item['content'],item['analysis'],item['answer'])
    #     print('-'*10)
    #     print(preprocess(item['content']),preprocess(item['analysis']),preprocess(item['answer']))
    #     input()
    # '''
    # print(preprocess('―-'))
    # '''